package com.zhiying.crawler.multiThreadClaw;


import com.zhiying.crawler.dto.BookDTO;
import com.zhiying.crawler.dto.ChapterDTO;
import com.zhiying.crawler.dto.SortDTO;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ClawUtil {

    private static final Logger logger = LoggerFactory.getLogger(ClawUtil.class);
    private static String PAGE_INDEX = "https://www.88dus.com";
    private static RequestConfig requestConfig;

    static {
        requestConfig = RequestConfig.custom().setConnectTimeout(1000*60*2)
                .setConnectionRequestTimeout(1000*60*2).setSocketTimeout(1000*60*2).build();
    }

    /**
     * 通过用户名和密码模拟登录
     * @return
     */
    public static HttpResponse mockLoginBbdus() {
        //your username and password for 88dus.com
        String userName = "";
        String password = "";
        CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).build();
        HttpPost httpPost = new HttpPost("https://www.88dus.com/");

        List<NameValuePair> nameValuePairs = new ArrayList<>();
        nameValuePairs.add(new BasicNameValuePair("username",userName));
        nameValuePairs.add(new BasicNameValuePair("password",password));

        HttpResponse response = null;
        try {
            httpPost.setEntity(new UrlEncodedFormEntity(nameValuePairs));
            response = httpClient.execute(httpPost);
            int status = response.getStatusLine().getStatusCode();
            if (status == 200 || status == 302 || status ==301) {
                return response;
            }
        } catch (UnsupportedEncodingException e) {
            logger.error("mock login 88 UnsupportedEncodingException:{}",e);
        } catch (IOException e){
            logger.error("mock login 88 io error:{}",e);
        }
        return null;
    }

    /**
     * 通过登陆后返回的身份信息访问网站的每一条link
     * @param response
     * @param url
     * @return
     */
    public static String accessByCookie(HttpResponse response, String url){
        HttpGet httpGet = new HttpGet(url);
        Header[] headers = response.getHeaders("Set-Cookie");
        for(Header header : headers){
            httpGet.addHeader(header);
        }
        CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).build();
        HttpResponse httpResponse = null;
        String html = null;
        try {
            httpResponse = httpClient.execute(httpGet);
            html = EntityUtils.toString(httpResponse.getEntity(),"gbk");
            return html;
        } catch (Exception e) {
            logger.info("error to get other page by login cookie:{}",e);
        }
        return null;
    }

    public static Map<String,String> accessByCookie2(HttpResponse response, String url){
        Map<String,String> htmlMap = new HashMap<>();
        HttpGet httpGet = new HttpGet(url);
        Header[] headers = response.getHeaders("Set-Cookie");
        for(Header header : headers){
            httpGet.addHeader(header);
        }
        CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).build();
        HttpResponse httpResponse = null;
        String html = null;
        try {
            httpResponse = httpClient.execute(httpGet);
            html = EntityUtils.toString(httpResponse.getEntity(),"gbk");
            htmlMap.put(url,html);
            return htmlMap;
        } catch (Exception e) {
            logger.info("error to get other page by login cookie:{}",e);
        }
        return null;
    }

    /**
     * 取出网站内分类的link 包括所有分类的所有页数
     * @param response
     * @return
     */
    public static List<SortDTO> getAllSortLinks(HttpResponse response){
        String indexHtml = ClawUtil.accessByCookie(response,PAGE_INDEX);
        List<String> sortList = new ArrayList();
        Document document = Jsoup.parse(indexHtml);
        Elements elements = document.select(".nav_l");
        Elements lis = elements.select("li");
        //获取所有分类第一页的link
        for(int i = 1; i < lis.size(); i++){
            Element li = lis.get(i);
            String url = li.select("a[href]").attr("href");
            sortList.add(PAGE_INDEX+url);
        }
        List<SortDTO> sortDTOS = new ArrayList<>();
        for(int i = 0; i < sortList.size(); i++){
            String pageOneLink = sortList.get(i);
            //爬取第一页 从而得到所有页数的link
            String sortPage1 = accessByCookie(response,pageOneLink);
            Document document1 = Jsoup.parse(sortPage1);
            String pageNum = document1.select("#pagestats").first().text();
            String totalPage = pageNum.split("/")[1];
            String tmp = pageOneLink.substring(0,(sortList.get(i).lastIndexOf("/"))-1);
            for(int j = 2; j < Integer.parseInt(totalPage); j++){
                String otherSortPageLink = tmp + j + "/";
                SortDTO sortDTO = new SortDTO(i+1,otherSortPageLink);
                sortDTOS.add(sortDTO);
            }
        }
        return sortDTOS;
    }

    /**
     * 取出每个分类下全部的书的link
     * @param sortPage 每一分类各页数的link
     * @return
     */
    public static List<BookDTO> getAllLinksPerSort(String sortPage, String sortLink){
        List<BookDTO> bookList = new ArrayList();
        Document document = Jsoup.parse(sortPage);
        Elements elements = document.select(".booklist");
        Elements lis = elements.select("li");
        for(int i = 1; i < lis.size(); i++){
            try {
                Element li = lis.get(i);
                Elements elementList = li.select("a[href]");

                Element urlElement = elementList.get(0);
                String url = urlElement.attr("href");
                String bookName = urlElement.text();

                Element authorElement = li.select(".zz").first();
                String author = authorElement.text();

                BookDTO bookDTO = new BookDTO(bookName, sortLink, PAGE_INDEX + url, author);
                bookList.add(bookDTO);
            }catch (Exception e){
                logger.error("parse sort html error,html: {},error: {}",sortPage,e);
            }
        }
        return bookList;
    }

    /**
     * 取出每本书的全部章节link
     * @param
     * @return
     */
    public static List<ChapterDTO> getAllLinksPerBook(BookDTO bookDTO){
        Document document = Jsoup.parse(bookDTO.getBookPage());
        Elements elements = document.select(".mulu");
        Elements lis = elements.select("li");
        List<ChapterDTO> chapterDTOS = new ArrayList<>();
        try {
            for (int i = 0; i < lis.size(); i++) {
                Element li = lis.get(i);
                String url = li.select("a[href]").first().attr("href");
                String chapter = li.select("a[href]").first().text();
                ChapterDTO chapterDTO = new ChapterDTO(chapter, bookDTO.getBookName(), null, bookDTO.getBookLink() + url, null);
                chapterDTOS.add(chapterDTO);
            }
        }catch (Exception e){
            logger.error("parse book page error:{}",e);
        }
        return chapterDTOS;
    }

    /**
     * html解析测试
     * @param args
     */
    public static void main(String[] args){
        HttpResponse response = ClawUtil.mockLoginBbdus();
        System.out.println(true);
        String index = ClawUtil.accessByCookie(response,"https://www.88dus.com");
        /*拿出书的分类*/
        /*Document document = Jsoup.parse(index);
        Elements elements = document.select(".nav_l");
        Elements lis1 = elements.select("li");
        for(int i = 0; i < lis1.size(); i++){
            Element li = lis1.get(i);
            String url = li.select("a[href]").attr("href");
            System.out.println(url);
        }*/
        /*以第一种分类为例，拿出第一种分类全部书单link*/
     /*   String sort1 = ClawUtil.accessByCookie(response,"https://www.88dus.com/sort1/1/");
        Document document1 = Jsoup.parse(sort1);
        Elements elements1 = document1.select(".booklist");
        Elements lis2 = elements1.select("li");
        for(int i = 1; i < lis2.size(); i++){
            Element li = lis2.get(i);
            String url = li.select("a[href]").first().attr("href");
            System.out.println(url);
            Element authorElement = li.select(".zz").first();
            String author = authorElement.text();
            System.out.println("author: "+author);
        }
        Element page = document1.select("#pagestats").first();
        String pageNum = page.text();
        System.out.println(pageNum);
        String totalPage = pageNum.split("/")[1];
        String sortLink1 = "https://www.88dus.com/sort1/1/";
        int pos = sortLink1.lastIndexOf("/");
        for(int i = 2; i < Integer.parseInt(totalPage); i++){
            String tmp = sortLink1.substring(0,pos-1);
            String sortMore = tmp + i + "/";
            System.out.println(sortMore);
        }*/

        /*以其中一本书为例，抓取该书已有的全部章节link 这些link拼接在分类的link上*/
        String fiction1 = ClawUtil.accessByCookie(response,"https://www.88dus.com/xiaoshuo/103/103886/");
        Document document2 = Jsoup.parse(fiction1);
        Elements elements2 = document2.select(".mulu");
        Elements lis3 = elements2.select("li");
        for(int i = 0; i < lis3.size(); i++){
            Element li = lis3.get(i);
            String url = li.select("a[href]").first().attr("href");
            String characterName = li.select("a[href]").first().text();
            System.out.println(url+" "+characterName);
        }
    }

}
